#!/usr/bin/python
# -*-coding:utf-8-*-
import json
import random
import time
from _socket import timeout
from urllib.error import HTTPError
from urllib.request import urlopen
from urllib.parse import unquote, quote
from urllib.request import Request
from urllib import parse
from bs4 import BeautifulSoup as bs
import pymysql.cursors
import os.path
import sys

from urllib3.util import timeout


class chongqing_tender:
    def __init__(self, tender_id=None, title=None, url=None, release_time=None, province_name="重庆市",
                 district_name=None, project_purchase_way=None, project_directory_name=None, buyer_name=None,
                 agent_name=None):
        self.tender_id = tender_id
        self.title = title
        self.url = url
        self.release_time = release_time
        self.province_name = province_name
        self.district_name = district_name
        self.project_purchase_way = project_purchase_way
        self.project_directory_name = project_directory_name
        self.buyer_name = buyer_name
        self.agent_name = agent_name


all_purchase_way = '[{"paramText":"公开招标","paramValue":100},{"paramText":"邀请招标","paramValue":200},' \
                   '{"paramText":"竞争性谈判","paramValue":300},{"paramText":"询价","paramValue":400},' \
                   '{"paramText":"单一来源","paramValue":500},{"paramText":"竞争性磋商","paramValue":800},' \
                   '{"paramText":"协议竞价","paramValue":6001},{"paramText":"网上询价","paramValue":6003}]'


def get_purchase_way(value):
    list_json = json.loads(all_purchase_way)
    for item in list_json:
        if item['paramValue'] == value:
            return item['paramText']
    return None


def get_list_page(page_index):
    """
    获取四川招标网的招标信息列表
    """
    # 获取数据连接
    connection = pymysql.connect(host="127.0.0.1", user="root", password="root", db="common", charset="utf8mb4")
    cursor = connection.cursor()

    url = "https://www.ccgp-chongqing.gov.cn/gwebsite/api/v1/notices/stable/new?pi=%s&ps=20" \
          "&startDate=2017-10-19" % page_index

    # 创建请求对象
    req = Request(url)
    req.add_header("Accept", "*/*")
    req.add_header("Accept-Encoding", "gzip, deflate, br")
    req.add_header("Accept-Language", "zh-CN,zh;q=0.9")
    req.add_header("Connection", "keep-alive")
    req.add_header("Host", "www.ccgp-chongqing.gov.cn")
    req.add_header("Referer", "https://www.ccgp-chongqing.gov.cn/notices/list")
    req.add_header("Sec-Fetch-Mode", "cors")
    req.add_header("Sec-Fetch-Site", "same-origin")
    req.add_header("User-Agent",
                   "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3941.4 Safari/537.36")
    req.add_header("X-Requested-With", "XMLHttpRequest")
    # 获得服务器返回的数据
    response = urlopen(req, timeout=6000)
    # 处理数据
    page = response.read().decode("utf-8")
    json_data = json.loads(page)
    list_json = json_data['notices']
    for item in list_json:
        tender = chongqing_tender()
        tender.tender_id = item['id']
        tender.title = item['title']
        tender.url = "https://www.ccgp-chongqing.gov.cn/notices/detail/%s" % item['id']
        tender.release_time = item['issueTime']
        if 'districtName' in item:
            tender.district_name = item['districtName']
        # 招标方式
        tender.project_purchase_way = get_purchase_way(item['projectPurchaseWay'])
        tender.project_directory_name = item['projectDirectoryName']
        tender.buyer_name = item['buyerName']
        if 'agentName' in item:
            tender.agent_name = item['agentName']
        print("id=%s,date=%s, title= %s ,purchase_way= %s,url= %s" % (
            tender.tender_id, tender.release_time, tender.title, tender.project_purchase_way,
            tender.url))

        # 创建sql语句
        sql = "REPLACE INTO `tender_info`(`tender_id`, `title`,`release_time`,`url`,`province_name`,`district_name`," \
              "`project_purchase_way`,`project_directory_name`,`buyer_name`,`agent_name`) " \
              "values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
        # 执行sql语句
        cursor.execute(sql, (tender.tender_id, tender.title, tender.release_time, tender.url, tender.province_name,
                             tender.district_name, tender.project_purchase_way, tender.project_directory_name,
                             tender.buyer_name, tender.agent_name))
        # 提交
        connection.commit()


if __name__ == '__main__':
    start = 305
    end = 6600
    for x in range(start, end):
        print("start page %s" % x)
        get_list_page(x)
        time.sleep(1)
